Logistic regression with theano¶
The 8th KIAS CAC Summer School
Machine Learning Practice I - Logistic Regression example
Author: Yung-Kyun Noh, Ph.D.
Logistic regression in Theano tutorial
Many parts are borrowed from Jiseob Kim's Github.
import theano
import theano.tensor as T
import numpy as np
import matplotlib.pyplot as plt
%pylab inline
floatX = theano.config.floatX
Data preparation¶
We use a two-class synthetic dataset from two different Gaussian density functions. \begin{eqnarray} p(\mathbf{x}|\mu_c, \Sigma_c) = \frac{1}{\sqrt{2\pi}|\Sigma_c|^\frac{1}{2}} \exp\left(-\frac{1}{2}(\mathbf{x} - \mu_c)^\top\Sigma_c^{-1}(\mathbf{x} - \mu_c)\right) \quad \text{for class} \quad c = {1,2} \end{eqnarray}
The first class uses $\mu_1 = \left(\begin{array}{c} 4 \\ 0 \end{array}\right)$, $\Sigma_1 = \left(\begin{array}{cc} 1 & 0 \\ 0 & 1 \end{array}\right)$, and the second class uses $\mu_2 = \left(\begin{array}{c} 0 \\ 4 \end{array}\right)$, $\Sigma_2 = \left(\begin{array}{cc} 1 & 0 \\ 0 & 1 \end{array}\right)$.
n_data = 50
data1 = np.random.multivariate_normal([4,0], [[1,0],[0,1]], n_data)
#print data1
data2 = np.random.multivariate_normal([0,4], [[1,0],[0,1]], n_data)
data_x = np.vstack([data1,data2])
data_y = np.hstack([np.ones((n_data,)), -np.ones((n_data,))])
shared_x = theano.shared(np.asarray(data_x, dtype=floatX), name='data_x')
shared_y = theano.shared(np.asarray(data_y, dtype=floatX), name='data_y')
w = theano.shared(np.ones((2,1), dtype=floatX), name='w')
b = theano.shared(np.zeros((1,), dtype=floatX), name='b')
def draw_state():
plt.rcParams['figure.figsize']=(5,5)
plt.scatter(data1[:,0],data1[:,1],30,'r')
plt.scatter(data2[:,0],data2[:,1],30,'b')
[x1min,x1max,x2min,x2max] = plt.axis()
x1val = np.arange(x1min,x1max,0.1)
wval = w.get_value(borrow=True)
bval = b.get_value(borrow=True)
plt.plot(x1val, -(wval[0]*x1val+bval)/wval[1], 'k')
plt.axis([x1min,x1max,x2min,x2max])
plt.show()
draw_state()
Logistic regression¶
The objective function $L$ with $\mathcal{D}=\{\mathbf{x}_i,y_i\}_{i = 1}^N$ \begin{eqnarray} L = -\log P(y_1,\ldots,y_N|\mathbf{x}_1,\ldots,\mathbf{x}_N; \mathbf{w}, b) + \lambda ||\mathbf{w}||^2 \quad \left( P(y_1,\ldots,y_N|\mathbf{x}_1,\ldots,\mathbf{x}N; \mathbf{w}, b) = \prod{i = 1}^N P(y = y_i|\mathbf{x} = \mathbf{x}_i; \mathbf{w}, b) \right) \end{eqnarray}
x = T.matrix('x')
y = T.vector('y')
p_y = 1/(1+T.exp(-(T.dot(x, w)+b)*T.reshape(y,(-1,1))))
loss = T.mean(-T.log(p_y)) + 0.01*w.norm(2)
Gradient Descent¶
wgrad = T.grad(loss, w)
bgrad = T.grad(loss, b)
lr = 0.1
#train = theano.function([], loss, givens=[(x,shared_x), (y,shared_y)], updates=[(w,w-lr*wgrad), (b,b-lr*bgrad)])
train = theano.function([x,y], loss, updates=[(w,w-lr*wgrad), (b,b-lr*bgrad)], allow_input_downcast=True)
import time
tic = time.clock()
for epoch in xrange(20):
# loss_val = train()
loss_val = train(data_x, data_y)
draw_state()
print('loss: {}, w norm: {}'.format(loss_val, np.sqrt(np.sum(w.get_value()**2))))
toc = time.clock()
toc - tic
